Skip to content

Commit

Permalink
In floatingip-list, use a memo for ports/routers/networks to reduce
Browse files Browse the repository at this point in the history 
round-trips in case of large # of floating IPs.

Change-Id: I4529bde36b130a228c568d1ec1d5747c4dea635e
Closes-Bug: 1673133
(cherry picked from commit ac281e5)
(cherry picked from commit 2554d5c)
  • Loading branch information
Hampapur Ajay committed Mar 21, 2017
1 parent 832f870 commit 61eaff2
Show file tree
Hide file tree
Showing 4 changed files with 243 additions and 45 deletions.
125 changes: 89 additions & 36 deletions src/config/vnc_openstack/vnc_openstack/neutron_plugin_db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,12 +606,6 @@ def _logical_router_delete(self, rtr_id):
self._raise_contrail_exception('RouterInUse', router_id=rtr_id)
#end _logical_router_delete

def _floatingip_list(self, obj_uuids=None, back_ref_id=None):
return self._vnc_lib.floating_ips_list(obj_uuids=obj_uuids,
back_ref_id=back_ref_id,
detail=True)
#end _floatingip_list

# find floating ip pools a project has access to
def _fip_pool_refs_project(self, project_id):
project_obj = self._project_read(proj_id=project_id)
Expand Down Expand Up @@ -1709,51 +1703,75 @@ def _floatingip_neutron_to_vnc(self, context, fip_q, oper):
return fip_obj
#end _floatingip_neutron_to_vnc

def _floatingip_vnc_to_neutron(self, fip_obj):
def _floatingip_vnc_to_neutron(self, fip_obj, memo_req=None):
fip_q_dict = {}

floating_net_id = self._vnc_lib.fq_name_to_id('virtual-network',
fip_obj.get_fq_name()[:-2])
try:
floating_net_id = memo_req['network_fqn'][tuple(fip_obj.get_fq_name()[:-2])]
except:
floating_net_id = self._vnc_lib.fq_name_to_id('virtual-network',
fip_obj.get_fq_name()[:-2])

tenant_id = fip_obj.get_project_refs()[0]['uuid'].replace('-', '')

port_id = None
router_id = None
port_obj = None
port_refs = fip_obj.get_virtual_machine_interface_refs()
if port_refs:
for port_ref in port_refs:

for port_ref in port_refs or []:
if memo_req:
try:
port_obj = memo_req['ports'][port_ref['uuid']]
except KeyError:
continue
else:
try:
port_obj = self._virtual_machine_interface_read(
port_id=port_ref['uuid'])
except NoIdError:
continue

# In case of floating ip on the Virtual-ip, svc-monitor will
# link floating ip to "right" interface of service VMs
# launched by ha-proxy service instance. Skip them
props = port_obj.get_virtual_machine_interface_properties()
if props:
interface_type = props.get_service_interface_type()
if interface_type == "right":
continue
# In case of floating ip on the Virtual-ip, svc-monitor will
# link floating ip to "right" interface of service VMs
# launched by ha-proxy service instance. Skip them
props = port_obj.get_virtual_machine_interface_properties()
if props:
interface_type = props.get_service_interface_type()
if interface_type == "right":
continue

port_id = port_ref['uuid']
break
except NoIdError:
pass
port_id = port_ref['uuid']
break

if port_obj:
port_net_id = port_obj.get_virtual_network_refs()[0]['uuid']
# find router_id from port
router_list = self._router_list_project(tenant_id, detail=True)
if memo_req:
router_list = memo_req['routers'].get(tenant_id, [])
else:
router_list = self._router_list_project(tenant_id, detail=True)

vmi_routers = {}
for router_obj in router_list or []:
for vmi in (router_obj.get_virtual_machine_interface_refs()
or []):
vmi_obj = self._virtual_machine_interface_read(
port_id=vmi['uuid'])
if (vmi_obj.get_virtual_network_refs()[0]['uuid'] ==
port_net_id):
router_id = router_obj.uuid
break
if router_id:
vmi_routers.update(dict((vmi_ref['uuid'], router_obj.uuid) for vmi_ref in (router_obj.get_virtual_machine_interface_refs() or [])))

if memo_req:
vmi_obj_list = [memo_req['ports'].get(vmi_id) for vmi_id in vmi_routers]
else:
if vmi_routers:
vmi_obj_list = self._virtual_machine_interface_list(
obj_uuids=vmi_routers.keys())
else:
vmi_obj_list = []

for vmi_obj in vmi_obj_list:
if vmi_obj is None:
continue

if (vmi_obj.get_virtual_network_refs()[0]['uuid'] ==
port_net_id):
router_id = vmi_routers[vmi_obj.uuid]
break

fip_q_dict['id'] = fip_obj.uuid
Expand Down Expand Up @@ -3507,9 +3525,44 @@ def floatingip_list(self, context, filters=None):
if not context['is_admin']:
backref_ids = [str(uuid.UUID(context['tenant']))]

fip_objs = self._floatingip_list(obj_uuids=fip_ids,
back_ref_id=backref_ids)
memo_req = {'routers': {},
'ports': {},
'network_fqn':{}}

fip_objs = self._vnc_lib.floating_ips_list(obj_uuids=fip_ids,
back_ref_id=backref_ids,
detail=True)

# prep memo for optimization
fip_vn_fqn = set(tuple(fip_obj.fq_name[:-2]) for fip_obj in fip_objs)
for vn_fqn in fip_vn_fqn:
try:
memo_req['network_fqn'][vn_fqn] = self._vnc_lib.fq_name_to_id(
'virtual-network', vn_fqn)
except NoIdError:
pass

fip_project_refs = list(set([fip_obj.get_project_refs()[0]['uuid']
for fip_obj in fip_objs]))
lr_objs = self._logical_router_list(parent_id=fip_project_refs)
for lr_obj in lr_objs:
tenant_id = lr_obj.parent_uuid.replace('-','')
try:
memo_req['routers'][tenant_id].append(lr_obj)
except KeyError:
memo_req['routers'][tenant_id] = [lr_obj]

vmi_uuids = []
for fip_obj in fip_objs:
vmi_uuids.extend([ref['uuid'] for ref in
fip_obj.get_virtual_machine_interface_refs() or []])
for lr_obj in lr_objs:
vmi_uuids.extend([ref['uuid'] for ref in
lr_obj.get_virtual_machine_interface_refs() or []])
vmi_objs = self._virtual_machine_interface_list(obj_uuids=vmi_uuids)
memo_req['ports'] = dict((vmi_obj.uuid, vmi_obj) for vmi_obj in vmi_objs)

# prepare result in neutron form and return
for fip_obj in fip_objs:
if 'floating_ip_address' in filters:
if (fip_obj.get_floating_ip_address() not in
Expand All @@ -3532,7 +3585,7 @@ def floatingip_list(self, context, filters=None):
continue

try:
ret_list.append(self._floatingip_vnc_to_neutron(fip_obj))
ret_list.append(self._floatingip_vnc_to_neutron(fip_obj, memo_req))
except NoIdError:
continue
except Exception as e:
Expand Down
8 changes: 8 additions & 0 deletions src/config/vnc_openstack/vnc_openstack/neutron_plugin_interface.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,14 @@ def _get_requests_data(self):
try:
if 'application/json' in ctype:
req = bottle.request.json
context = req['context']
if context.get('tenant') and context.get('tenant_id'):
if context['tenant'] != context['tenant_id']:
bottle.abort(400, 'Unable to parse request data')
elif context.get('tenant'):
context['tenant_id'] = context['tenant']
elif context.get('tenant_id'):
context['tenant'] = context['tenant_id']
return req['context'], req['data']
except Exception as e:
bottle.abort(400, 'Unable to parse request data')
Expand Down
145 changes: 136 additions & 9 deletions src/config/vnc_openstack/vnc_openstack/tests/test_basic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ def read_resource(self, url_pfx, id):
# end read_resource

def list_resource(self, url_pfx,
proj_uuid=None, req_fields=None, req_filters=None):
proj_uuid=None, req_fields=None, req_filters=None,
is_admin=False):
if proj_uuid == None:
proj_uuid = self._vnc_lib.fq_name_to_id('project',
fq_name=['default-domain', 'default-project'])
Expand All @@ -44,7 +45,7 @@ def list_resource(self, url_pfx,
'user_id': '',
'tenant_id': proj_uuid,
'roles': '',
'is_admin': 'False'}
'is_admin': is_admin}
data = {'fields': req_fields, 'filters': req_filters or {}}
body = {'context': context, 'data': data}
resp = self._api_svr_app.post_json(
Expand All @@ -53,10 +54,10 @@ def list_resource(self, url_pfx,
# end list_resource

def create_resource(self, res_type, proj_id, name=None,
extra_res_fields=None):
extra_res_fields=None, is_admin=False):
context = {'operation': 'CREATE',
'user_id': '',
'is_admin': False,
'is_admin': is_admin,
'roles': '',
'tenant_id': proj_id}
if name:
Expand All @@ -72,10 +73,10 @@ def create_resource(self, res_type, proj_id, name=None,
resp = self._api_svr_app.post_json('/neutron/%s' %(res_type), body)
return json.loads(resp.text)

def delete_resource(self, res_type, proj_id, id):
def delete_resource(self, res_type, proj_id, id, is_admin=False):
context = {'operation': 'DELETE',
'user_id': '',
'is_admin': False,
'is_admin': is_admin,
'roles': '',
'tenant_id': proj_id}

Expand All @@ -84,10 +85,11 @@ def delete_resource(self, res_type, proj_id, id):
body = {'context': context, 'data': data}
self._api_svr_app.post_json('/neutron/%s' %(res_type), body)

def update_resource(self, res_type, res_id, proj_id, name=None, extra_res_fields=None):
context = {'operation': 'UPDATE',
def update_resource(self, res_type, res_id, proj_id, name=None, extra_res_fields=None,
is_admin=False, operation=None):
context = {'operation': operation or 'UPDATE',
'user_id': '',
'is_admin': False,
'is_admin': is_admin,
'roles': '',
'tenant_id': proj_id}
if name:
Expand Down Expand Up @@ -536,8 +538,133 @@ def test_update_port_with_security_group_and_port_security_disabled(self):
sg_q = self.create_resource('security_group', proj_obj.uuid)
with ExpectedException(webtest.app.AppError):
self.update_resource('port', port_q['id'], proj_obj.uuid, extra_res_fields={'security_groups': [sg_q['id']]})

def test_floating_ip_list(self):
proj_objs = []
for i in range(3):
proj_id = str(uuid.uuid4())
proj_name = 'proj-%s-%s' %(self.id(), i)
test_case.get_keystone_client().tenants.add_tenant(proj_id, proj_name)
proj_objs.append(self._vnc_lib.project_read(id=proj_id))

sg_q_list = [self.create_resource('security_group', proj_objs[i].uuid)
for i in range(3)]

# public network on last project
pub_net1_q = self.create_resource('network', proj_objs[-1].uuid,
name='public-network-%s-1' %(self.id()),
extra_res_fields={'router:external': True})
self.create_resource('subnet', proj_objs[-1].uuid,
name='public-subnet-%s-1' %(self.id()),
extra_res_fields={
'network_id': pub_net1_q['id'],
'cidr': '10.1.1.0/24',
'ip_version': 4,
})
pub_net2_q = self.create_resource('network', proj_objs[-1].uuid,
name='public-network-%s-2' %(self.id()),
extra_res_fields={'router:external': True})
self.create_resource('subnet', proj_objs[-1].uuid,
name='public-subnet-%s-2' %(self.id()),
extra_res_fields={
'network_id': pub_net2_q['id'],
'cidr': '20.1.1.0/24',
'ip_version': 4,
})

def create_net_subnet_port_assoc_fip(i, pub_net_q_list,
has_routers=True):
net_q_list = [self.create_resource('network', proj_objs[i].uuid,
name='network-%s-%s-%s' %(self.id(), i, j)) for j in range(2)]
subnet_q_list = [self.create_resource('subnet', proj_objs[i].uuid,
name='subnet-%s-%s-%s' %(self.id(), i, j),
extra_res_fields={
'network_id': net_q_list[j]['id'],
'cidr': '1.%s.%s.0/24' %(i, j),
'ip_version': 4,
}) for j in range(2)]

if has_routers:
router_q_list = [self.create_resource('router', proj_objs[i].uuid,
name='router-%s-%s-%s' %(self.id(), i, j),
extra_res_fields={
'external_gateway_info': {
'network_id': pub_net_q_list[j]['id'],
}
}) for j in range(2)]
[self.update_resource('router', router_q_list[j]['id'],
proj_objs[i].uuid, is_admin=True, operation='ADDINTERFACE',
extra_res_fields={'subnet_id': subnet_q_list[j]['id']})
for j in range(2)]
else:
router_q_list = None

port_q_list = [self.create_resource('port', proj_objs[i].uuid,
name='port-%s-%s-%s' %(self.id(), i, j),
extra_res_fields={
'network_id': net_q_list[j]['id'],
'security_groups': [sg_q_list[i]['id']],
}) for j in range(2)]

fip_q_list = [self.create_resource('floatingip', proj_objs[i].uuid,
name='fip-%s-%s-%s' %(self.id(), i, j),
is_admin=True,
extra_res_fields={'floating_network_id': pub_net_q_list[j]['id'],
'port_id': port_q_list[j]['id']}) for j in range(2)]

return {'network': net_q_list, 'subnet': subnet_q_list,
'ports': port_q_list, 'fips': fip_q_list,
'routers': router_q_list}
# end create_net_subnet_port_assoc_fip

created = []
# without routers
created.append(create_net_subnet_port_assoc_fip(
0, [pub_net1_q, pub_net2_q], has_routers=False))

# with routers
created.append(create_net_subnet_port_assoc_fip(
1, [pub_net1_q, pub_net2_q], has_routers=True))

# 1. list as admin for all routers
fip_dicts = self.list_resource('floatingip', is_admin=True)
# convert list to dict by id
fip_dicts = dict((fip['id'], fip) for fip in fip_dicts)
# assert all floatingip we created recevied back
for fip in created[0]['fips'] + created[1]['fips']:
self.assertIn(fip['id'], fip_dicts.keys())

# assert router-id present in fips of proj[1]
self.assertEqual(created[1]['routers'][0]['id'],
fip_dicts[created[1]['fips'][0]['id']]['router_id'])
self.assertEqual(created[1]['routers'][1]['id'],
fip_dicts[created[1]['fips'][1]['id']]['router_id'])

# assert router-id not present in fips of proj[0]
self.assertEqual(None,
fip_dicts[created[0]['fips'][0]['id']]['router_id'])
self.assertEqual(None,
fip_dicts[created[0]['fips'][1]['id']]['router_id'])

# 2. list routers within project
fip_dicts = self.list_resource(
'floatingip', proj_uuid=proj_objs[0].uuid)
self.assertEqual(None,
fip_dicts[0]['router_id'])
self.assertEqual(None,
fip_dicts[1]['router_id'])
# convert list to dict by port-id
fip_dicts = dict((fip['port_id'], fip) for fip in fip_dicts)
# assert fips point to right port
self.assertEqual(created[0]['ports'][0]['fixed_ips'][0]['ip_address'],
fip_dicts[created[0]['ports'][0]['id']]['fixed_ip_address'])
self.assertEqual(created[0]['ports'][1]['fixed_ips'][0]['ip_address'],
fip_dicts[created[0]['ports'][1]['id']]['fixed_ip_address'])
# end test_floating_ip_list

# end class TestBasic


class TestExtraFieldsPresenceByKnob(test_case.NeutronBackendTestCase):
@classmethod
def setUpClass(cls):
Expand Down

0 comments on commit 61eaff2

Please sign in to comment.