Merge "Enhancements to support some flow test cases" into R3.0

common/flow_tests/base.py

@@ -0,0 +1,42 @@
+import re
+
+from common.neutron.base import BaseNeutronTest
+
+class FlowTestBase(BaseNeutronTest):
+
+    @classmethod
+    def setUpClass(cls):
+        super(FlowTestBase, cls).setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        super(FlowTestBase, cls).tearDownClass()
+    # end tearDownClass
+
+
+    def get_flows_exported(self, agent_name, last="1m"):
+        ''' agent_name is of format nodek1:Compute:contrail-vrouter-agent:0
+        '''
+        # TODO
+        # Use test-code's api to do query instead of running contrail-stats cmd
+        cmd = '''contrail-stats --table SandeshMessageStat.msg_info --select \
+"SUM(msg_info.messages)" --last %s --where \
+'name=%s' 'msg_info.type=FlowLogDataObject' | tail -1'''  % (last, agent_name)
+        output = self.inputs.run_cmd_on_server(self.inputs.collector_ips[0], cmd)
+        digits = re.findall('\d+', output)
+        if digits:
+            return digits[0]
+        else:
+            self.logger.debug('No flows seen in collector for cmd %s' % (cmd))
+            return None
+    # end get_flows_exported
+
+    def setup_flow_export_rate(self, value):
+        ''' Set flow export rate and handle the cleanup
+        '''
+        vnc_lib_fixture = self.connections.vnc_lib_fixture
+        current_rate = vnc_lib_fixture.get_flow_export_rate()
+        vnc_lib_fixture.set_flow_export_rate(value)
+        self.addCleanup(vnc_lib_fixture.set_flow_export_rate, current_rate)
+    # end setup_flow_export_rate
+

common/neutron/base.py

@@ -52,42 +52,71 @@ def tearDownClass(cls):
         super(BaseNeutronTest, cls).tearDownClass()
     # end tearDownClass
 
-    def create_vn(self, vn_name=None, vn_subnets=None, vxlan_id=None,
-        enable_dhcp=True, cleanup=True):
+    @classmethod
+    def create_only_vn(cls, vn_name=None, vn_subnets=None, vxlan_id=None,
+                   enable_dhcp=True):
+        '''Classmethod to do only VN creation
+        '''
         if not vn_name:
             vn_name = get_random_name('vn')
         if not vn_subnets:
             vn_subnets = [get_random_cidr()]
-        vn_fixture = VNFixture(project_name=self.inputs.project_name,
-                      connections=self.connections,
-                      inputs=self.inputs,
+        vn_fixture = VNFixture(project_name=cls.inputs.project_name,
+                      connections=cls.connections,
+                      inputs=cls.inputs,
                       vn_name=vn_name,
                       subnets=vn_subnets,
                       vxlan_id=vxlan_id,
                       enable_dhcp=enable_dhcp)
         vn_fixture.setUp()
+        return vn_fixture
+    # end create_only_vn
+
+
+    def create_vn(self, vn_name=None, vn_subnets=None, vxlan_id=None,
+        enable_dhcp=True, cleanup=True):
+        vn_fixture = self.create_only_vn(vn_name=vn_name,
+                                     vn_subnets=vn_subnets,
+                                     vxlan_id=vxlan_id,
+                                     enable_dhcp=enable_dhcp)
         if cleanup:
             self.addCleanup(vn_fixture.cleanUp)
 
         return vn_fixture
     # end create_vn
 
-    def create_vm(self, vn_fixture, vm_name=None, node_name=None,
+    @classmethod
+    def create_only_vm(cls, vn_fixture, vm_name=None, node_name=None,
                   flavor='contrail_flavor_small',
                   image_name='ubuntu-traffic',
                   port_ids=[]):
         if not vm_name:
             vm_name = 'vm-%s' % (get_random_name(vn_fixture.vn_name))
-        return self.useFixture(
-            VMFixture(
-                project_name=self.inputs.project_name,
-                connections=self.connections,
-                vn_obj=vn_fixture.obj,
-                vm_name=vm_name,
-                image_name=image_name,
-                flavor=flavor,
-                node_name=node_name,
-                port_ids=port_ids))
+        vm_obj = VMFixture(
+                    project_name=cls.inputs.project_name,
+                    connections=cls.connections,
+                    vn_obj=vn_fixture.obj,
+                    vm_name=vm_name,
+                    image_name=image_name,
+                    flavor=flavor,
+                    node_name=node_name,
+                    port_ids=port_ids)
+        vm_obj.setUp()
+        return vm_obj
+    # end create_only_vm
+
+    def create_vm(self, vn_fixture, vm_name=None, node_name=None,
+                  flavor='contrail_flavor_small',
+                  image_name='ubuntu-traffic',
+                  port_ids=[]):
+        vm_fixture = self.create_only_vm(vn_fixture,
+                        vm_name=vm_name,
+                        node_name=node_name,
+                        flavor=flavor,
+                        image_name=image_name,
+                        port_ids=port_ids)
+        self.addCleanup(vm_fixture.cleanUp)
+        return vm_fixture
 
     def create_router(self, router_name, tenant_id=None):
         obj = self.quantum_h.create_router(router_name, tenant_id)

fixtures/compute_node_test.py

@@ -69,6 +69,7 @@ def __init__(
         self.max_system_flows = 512000
         self.agent_inspect_h = self.connections.agent_inspect[self.ip]
         self.flow_table = None
+        self.agent_generator_name = None
     # end __init__
 
     def setUp(self):
@@ -257,6 +258,14 @@ def sup_vrouter_process_restart(self):
             "Wait for contrail-vrouter-agent to be in active state.")
         self.wait_for_vrouter_agent_state(state='active')
 
+    def restart_service(self, service_name):
+        ''' Restart any contrail service on this compute node
+        '''
+        self.inputs.restart_service(service_name, [self.ip])
+
+    def restart_agent(self):
+        self.restart_service('contrail-vrouter-agent')
+
     def sup_vrouter_process_start(self):
         self.logger.info(
             'start supervisor-vrouter process in node %s' %
@@ -590,3 +599,11 @@ def setup_vrouter_module_params(self, params):
         self.addCleanup(self.del_vrouter_module_params, params,
             reload_vrouter=True)
     # end setup_vrouter_module_params
+
+    def get_agent_generator_name(self):
+        if not self.agent_generator_name:
+            self.agent_generator_name = self.agent_inspect_h.get_generator_name()
+        return self.agent_generator_name
+    # end get_agent_generator_name
+
+

fixtures/vm_test.py

@@ -692,15 +692,15 @@ def get_agent_label(self):
                 self.agent_label[vn_fq_name] = self.get_tap_intf_of_vmi(vmi)['label']
         return self.agent_label
 
-    def get_local_ips(self):
-        if not getattr(self, 'local_ips', None):
+    def get_local_ips(self, refresh=False):
+        if refresh or not getattr(self, 'local_ips', None):
             for (vn_fq_name, vmi) in self.get_vmi_ids().iteritems():
                 self.local_ips[vn_fq_name] = self.get_tap_intf_of_vmi(vmi)['mdata_ip_addr']
         return self.local_ips
 
     def get_local_ip(self, refresh=False):
         if refresh or not getattr(self, '_local_ip', None):
-            local_ips = self.get_local_ips()
+            local_ips = self.get_local_ips(refresh=refresh)
             for vn_fq_name in self.vn_fq_names:
                 if  self.vnc_lib_fixture.get_active_forwarding_mode(vn_fq_name) =='l2':
                     self.logger.debug("skipping ping to one of the 169.254.x.x IPs")
@@ -2268,6 +2268,7 @@ def provision_static_route(
         else:
             self.logger.info("%s" % (stdout))
 
+
     def _gather_details(self):
         self.cs_vmi_obj = {}
         self.get_vmi_objs()

fixtures/vnc_api_test.py

@@ -228,5 +228,29 @@ def set_global_forwarding_mode(self,forwarding_mode):
         self.vnc_api_h.global_vrouter_config_update(gsc_obj)
     #end set_global_forwarding_mode
 
+    def get_flow_export_rate(self):
+        fq_name = [ 'default-global-system-config',
+                    'default-global-vrouter-config']
+        gv_obj = self.vnc_api_h.global_vrouter_config_read(fq_name=fq_name)
+        rate = gv_obj.get_flow_export_rate()
+        if not rate:
+            # If not set, return 100 , since this is default
+            return 100
+        else:
+            return rate
+    # end get_flow_export_rate
 
+    def set_flow_export_rate(self, value):
+        '''
+        Set flow export rate in default global vrouter config
+        value : Value of flow export rate to be set
+        '''
+        fq_name = [ 'default-global-system-config',
+                    'default-global-vrouter-config']
+        gv_obj = self.vnc_api_h.global_vrouter_config_read(fq_name=fq_name)
+        gv_obj.set_flow_export_rate(int(value))
+        self.vnc_api_h.global_vrouter_config_update(gv_obj)
+        self.logger.info('Setting flow export rate: %s' % (value))
+        return True
+    # end set_flow_export_rate
 # end VncLibFixture

tcutils/agent/vna_introspect_utils.py

@@ -760,6 +760,14 @@ def get_acls_list(self):
             l.append(p)
         return l
 
+    def get_generator_name(self):
+        ''' Returns string of format nodek1:Compute:contrail-vrouter-agent:0
+        '''
+        xml_data = self.dict_get('Snh_SandeshUVECacheReq?x=ModuleClientState')
+        name = xml_data.getchildren()[0].xpath('./data/ModuleClientState/name')[0].text
+        return name
+    # end get_generator_name
+
 if __name__ == '__main__':
 
     vvnagnt = AgentInspect('10.204.217.12')

tcutils/fabutils.py

@@ -80,7 +80,7 @@ def remote_cmd(host_string, cmd, password=None, gateway=None,
     _run = sudo if with_sudo else run
 
     # with hide('everything'), settings(host_string=host_string,
-    with settings(
+    with hide('everything'), settings(
             host_string=host_string,
             gateway=gateway,
             warn_only=warn_only,
@@ -94,8 +94,8 @@ def remote_cmd(host_string, cmd, password=None, gateway=None,
         while tries > 0:
             try:
                 output = _run(cmd, timeout=timeout, **kwargs)
-            except (CommandTimeout, NetworkError):
-                pass
+            except (CommandTimeout, NetworkError) as e:
+                log.warn('Unable to run command %s: %s' % (cmd, str(e)))
                 tries -= 1
                 time.sleep(5)
                 continue

tcutils/traffic_utils/hping_traffic.py

@@ -102,6 +102,8 @@ def parse_result_file(self, result_file):
         round-trip min/avg/max = 0.1/5.8/1012.5 ms
 
         '''
+        reg_result = None
+        rtt_result = None
         result_data = {'sent': None, 'received': None, 'loss': None,
                        'rtt_min':None, 'rtt_avg':None, 'rtt_max':None}
         search1 = '''(\S+) packets transmitted, (\S+) packets received, (\S+)% packet loss'''
@@ -110,16 +112,17 @@ def parse_result_file(self, result_file):
 #            result_log = myfile.read()
         cmds = ['cat %s' %(self.result_file),
                 'cat %s' %(self.log_file)]
-        result = self.sender_vm_fixture.run_cmd_on_vm(cmds)
+        result = self.sender_vm_fixture.run_cmd_on_vm(cmds, timeout=300)
 
         result_content = result[cmds[0]]
         result_log = result[cmds[1]]
-        reg_result = re.search(search1, result_content)
+        if result_content:
+            reg_result = re.search(search1, result_content)
+            rtt_result = re.search(search2, result_content)
         if reg_result:
             result_data['sent'] = reg_result.group(1)
             result_data['received'] = reg_result.group(2)
             result_data['loss'] = reg_result.group(3)
-        rtt_result = re.search(search1, result_content)
         if rtt_result:
             result_data['rtt_min'] = rtt_result.group(1)
             result_data['rtt_avg'] = rtt_result.group(2)

tcutils/util.py

@@ -756,6 +756,7 @@ def wrapper(*args, **kwargs):
 
 def run_cmd_on_server(issue_cmd, server_ip, username,
                       password, pty=True, as_sudo=False):
+    log.debug('[%s]: Running cmd : %s' % (server_ip, issue_cmd))
     with hide('everything'):
         with settings(
             host_string='%s@%s' % (username, server_ip), password=password,
@@ -764,6 +765,7 @@ def run_cmd_on_server(issue_cmd, server_ip, username,
                 output = sudo('%s' % (issue_cmd), pty=pty) 
             else:
                 output = run('%s' % (issue_cmd), pty=pty)
+            log.debug('Output : %s' % (output))
             return output
 # end run_cmd_on_server
 
@@ -990,3 +992,22 @@ def get_build_sku(openstack_node_ip, openstack_node_password='c0ntrail123', user
                 pass
             tries -= 1
         return build_sku
+
+def is_almost_same(val1, val2, threshold_percent=10, num_type=int):
+    ''' returns false if val2 is less than or greater than threshold_percent
+        percent of val1
+    '''
+
+    val1 = num_type(val1)
+    val2 = num_type(val2)
+    if val1:
+        if (abs(float(val1-val2))/val1)*100 < threshold_percent:
+            return True
+        else:
+            return False
+    else:
+        if val2:
+            return False
+        else:
+            return True
+# end is_almost_same

test_v1.py

@@ -1,5 +1,6 @@
 from common.isolated_creds import *
 from test import BaseTestCase
+import time
 
 class BaseTestCase_v1(BaseTestCase):
     @classmethod
@@ -53,5 +54,10 @@ def tearDownClass(cls):
             cls.admin_isolated_creds.delete_user(cls.isolated_creds.username)
         super(BaseTestCase_v1, cls).tearDownClass()
     # end tearDownClass
+
+    def sleep(self, value):
+        self.logger.debug('Sleeping for %s seconds..' % (value))
+        time.sleep(value)
+    # end sleep