From b152b74f3b63e765b422be4e3e19b499032c7634 Mon Sep 17 00:00:00 2001 From: dineshb-jnpr Date: Sat, 21 Jan 2017 23:08:17 -0800 Subject: [PATCH] Allocate cluster service n/w fip for pods in Isolated Namespaces. Pods created in K8s isolated namespaces need to be assigned a fip, from k8s cluster service network, so that these pods can talk to k8s cluster services. These code changes are for the following: 1. Allocate a fip from k8s cluster service n/w to vmi of pods, that are created in "isolated" k8s namespaces. 2. Api-server changes to support floating-ip-pool create for a subnet on a virtual-network. 3. Api-server changes to support floating-ip alloc from a subnet pointed to by its floating-ip-pool. 4. Unit test to verify api-server changes. Change-Id: I90d508c28f191ed04786d6c259a1aefd4202b44b Partial-Bug: #1639062 --- src/config/api-server/tests/test_ip_alloc.py | 235 ++++++++++++++++++ src/config/api-server/vnc_addr_mgmt.py | 1 + src/config/api-server/vnc_cfg_types.py | 137 +++++++++- src/config/api-server/vnc_db.py | 20 +- src/container/kube-manager/SConscript | 2 + .../kube_manager/vnc/config_db.py | 2 + .../kube_manager/vnc/vnc_kubernetes.py | 86 +++++-- .../kube_manager/vnc/vnc_namespace.py | 20 +- .../kube-manager/kube_manager/vnc/vnc_pod.py | 3 +- 9 files changed, 463 insertions(+), 43 deletions(-) diff --git a/src/config/api-server/tests/test_ip_alloc.py b/src/config/api-server/tests/test_ip_alloc.py index fc90d1817b6..386efb2e3a5 100644 --- a/src/config/api-server/tests/test_ip_alloc.py +++ b/src/config/api-server/tests/test_ip_alloc.py @@ -9,6 +9,7 @@ import uuid import logging import coverage +import ipaddress import testtools from testtools.matchers import Equals, MismatchError, Not, Contains @@ -2268,6 +2269,240 @@ def test_ip_alloc_clash(self): isolated_vmi_obj) # end test_ip_alloc_clash + def test_floating_ip_pool_subnet_create(self): + """ + Test Floating-Ip-Pool create by specifiying subnet uuid in an Ipam of + a virtual-network. This testcase covers both valid and invalid subnet + uuid specification scenarios. + """ + def _get_ipam_subnet_uuid(ipam_ref, ipam_obj): + """ + Convenience function to return subnet uuid in a ipam ref on + a virtual-network. + """ + if ipam_ref['to'] == ipam_obj.get_fq_name(): + ipam_subnets = ipam_ref['attr'].get_ipam_subnets() + if ipam_subnets: + # We will use the first subnet in the matching IPAM. + return True, ipam_subnets[0].get_subnet_uuid() + return False, None + #end _get_ipam_subnet_uuid + + # Create a test project. + proj_obj = Project('proj-%s' %(self.id()), parent_obj=Domain()) + self._vnc_lib.project_create(proj_obj) + + # + # User-defined subnet. + # + + # Create a user-defined subnet IPAM. + ipam_obj = NetworkIpam('user-subnet-ipam-%s' %(self.id()), proj_obj) + self._vnc_lib.network_ipam_create(ipam_obj) + ipam_obj = self._vnc_lib.network_ipam_read(\ + fq_name=ipam_obj.get_fq_name()) + + # Create a virtual-network and add user-defined IPAM to it. + vn_name = 'user-vn-%s' %(self.id()) + vn_obj = VirtualNetwork(vn_name, proj_obj) + ipam_sn_v4 = IpamSubnetType(subnet=SubnetType('11.1.1.0', 24)) + vn_obj.add_network_ipam(ipam_obj, VnSubnetsType([ipam_sn_v4])) + self._vnc_lib.virtual_network_create(vn_obj) + vn_obj = self._vnc_lib.virtual_network_read(\ + fq_name=vn_obj.get_fq_name()) + + # Create a Floating-ip-pool that points to an "invalid subnet id + # on the virtual-network. Verify, floatin-ip-pool create fails. + fip_subnets = FloatingIpPoolSubnetType(subnet_uuid=[self.id()]) + fip_pool_obj = FloatingIpPool( + 'user-fip-pool-%s' %(self.id()), parent_obj=vn_obj, + floating_ip_pool_subnets = fip_subnets) + err_msg = "Subnet %s was not found in virtual-network %s" %\ + (self.id(), vn_obj.get_uuid()) + with ExpectedException(cfgm_common.exceptions.BadRequest, + err_msg) as e: + self._vnc_lib.floating_ip_pool_create(fip_pool_obj) + + # Get subnet uuid of user defined ipam in the virtual-network. + ipam_refs = vn_obj.get_network_ipam_refs() + svc_subnet_uuid = None + for ipam_ref in ipam_refs: + found, svc_subnet_uuid = _get_ipam_subnet_uuid(ipam_ref, ipam_obj) + if found: + break + + # Create a Floating-ip-pool that points to an subnet id of the + # user-define ipam. Verify, floatin-ip-pool create succeeds. + fip_subnets = FloatingIpPoolSubnetType(subnet_uuid = [svc_subnet_uuid]) + fip_pool_obj = FloatingIpPool( + 'user-fip-pool-%s' %(self.id()), parent_obj=vn_obj, + floating_ip_pool_subnets = fip_subnets) + self._vnc_lib.floating_ip_pool_create(fip_pool_obj) + + # + # Flat subnet. + # + + # Create a flat subnet IPAM. + ipam_sn_v4 = [IpamSubnetType(subnet=SubnetType('11.1.1.0', 24))] + ipam_obj = NetworkIpam('flat-subnet-ipam-%s' %(self.id()), proj_obj) + ipam_obj.set_ipam_subnet_method('flat-subnet') + ipam_obj.set_ipam_subnets(IpamSubnets(ipam_sn_v4)) + self._vnc_lib.network_ipam_create(ipam_obj) + ipam_obj = self._vnc_lib.network_ipam_read(\ + fq_name=ipam_obj.get_fq_name()) + + # Create a virtual-network and add flat-subnet IPAM to it. + vn_name = 'flat-vn-%s' %(self.id()) + vn_obj = VirtualNetwork(vn_name, proj_obj) + vn_obj.add_network_ipam(ipam_obj, VnSubnetsType([])) + vn_obj.set_virtual_network_properties(\ + VirtualNetworkType(forwarding_mode='l3')) + self._vnc_lib.virtual_network_create(vn_obj) + vn_obj = self._vnc_lib.virtual_network_read(\ + fq_name=vn_obj.get_fq_name()) + + # Create a Floating-ip-pool that points to an "invalid" subnet id + # on the virtual-network. Verify, floatin-ip-pool create fails. + fip_subnets = FloatingIpPoolSubnetType(subnet_uuid = [self.id()]) + fip_pool_obj = FloatingIpPool( + 'flat-fip-pool-%s' %(self.id()), parent_obj=vn_obj, + floating_ip_pool_subnets = fip_subnets) + err_msg = "Subnet %s was not found in virtual-network %s" %\ + (self.id(), vn_obj.get_uuid()) + with ExpectedException(cfgm_common.exceptions.BadRequest, + err_msg) as e: + self._vnc_lib.floating_ip_pool_create(fip_pool_obj) + + # Get subnet uuid of flat subnet ipam in the virtual-network. + ipam_refs = vn_obj.get_network_ipam_refs() + svc_subnet_uuid = None + for ipam_ref in ipam_refs: + found, svc_subnet_uuid = _get_ipam_subnet_uuid(ipam_ref, ipam_obj) + if found: + break + + # Create a Floating-ip-pool that points to an subnet id of the + # flat-subnet ipam. Verify, floatin-ip-pool create succeeds. + fip_subnets = FloatingIpPoolSubnetType(subnet_uuid = [svc_subnet_uuid]) + fip_pool_obj = FloatingIpPool( + 'flat-fip-pool-%s' %(self.id()), parent_obj=vn_obj, + floating_ip_pool_subnets = fip_subnets) + self._vnc_lib.floating_ip_pool_create(fip_pool_obj) + + # end test_floating_ip_pool_subnet_create + + def test_floating_ip_alloc(self): + """ + Test Floating-Ip create from floating-ip-pools that have been + created by requesting a specific subnet. Verify that the floating-ip + created is from the corresponding subnet. + """ + def _get_ipam_subnet_uuid(ipam_ref, ipam_obj): + """ + Convenience function to return subnet uuid in a ipam ref on + a virtual-network. + """ + if ipam_ref['to'] == ipam_obj.get_fq_name(): + ipam_subnets = ipam_ref['attr'].get_ipam_subnets() + if ipam_subnets: + # We will use the first subnet in the matching IPAM. + return True, ipam_subnets[0].get_subnet_uuid() + return False, None + #end _get_ipam_subnet_uuid + + # Create a test project. + proj_obj = Project('proj-%s' %(self.id()), parent_obj=Domain()) + self._vnc_lib.project_create(proj_obj) + + vn_name = 'vn-%s' %(self.id()) + vn_obj = VirtualNetwork(vn_name, proj_obj) + + # Create a user-defined subnet IPAM. + user_ipam_obj = NetworkIpam('user-subnet-ipam-%s' %(self.id()), proj_obj) + self._vnc_lib.network_ipam_create(user_ipam_obj) + user_ipam_obj = self._vnc_lib.network_ipam_read(\ + fq_name=user_ipam_obj.get_fq_name()) + + # Add user-defined subnet ipam to virtual-network. + ipam_sn_v4 = IpamSubnetType(subnet=SubnetType('10.1.1.0', 24)) + vn_obj.add_network_ipam(user_ipam_obj, VnSubnetsType([ipam_sn_v4])) + + # Create a flat subnet IPAM. + ipam_sn_v4 = [IpamSubnetType(subnet=SubnetType('11.1.1.0', 24))] + flat_ipam_obj = NetworkIpam(\ + 'flat-subnet-ipam-%s' %(self.id()), proj_obj) + flat_ipam_obj.set_ipam_subnet_method('flat-subnet') + flat_ipam_obj.set_ipam_subnets(IpamSubnets(ipam_sn_v4)) + self._vnc_lib.network_ipam_create(flat_ipam_obj) + flat_ipam_obj = self._vnc_lib.network_ipam_read(\ + fq_name=flat_ipam_obj.get_fq_name()) + + # Add flat subnet ipam to virtual-network. + vn_obj.add_network_ipam(flat_ipam_obj, VnSubnetsType([])) + vn_obj.set_virtual_network_properties(\ + VirtualNetworkType(forwarding_mode='l3')) + + # Create the virtual-network. + self._vnc_lib.virtual_network_create(vn_obj) + vn_obj = self._vnc_lib.virtual_network_read(\ + fq_name=vn_obj.get_fq_name()) + + # Get subnet uuid of user defined ipam in the virtual-network. + ipam_refs = vn_obj.get_network_ipam_refs() + user_subnet_uuid = None + for ipam_ref in ipam_refs: + found, user_subnet_uuid = _get_ipam_subnet_uuid(ipam_ref, + user_ipam_obj) + if found: + break + + # Get subnet uuid of flat subnet ipam in the virtual-network. + flat_subnet_uuid = None + for ipam_ref in ipam_refs: + found, flat_subnet_uuid = _get_ipam_subnet_uuid(ipam_ref, + flat_ipam_obj) + if found: + break + + # Create a Floating-ip-pool that points to the user-defined subnet id. + # Verify, floating-ip-pool create succeeds. + fip_subnets = FloatingIpPoolSubnetType(subnet_uuid=[user_subnet_uuid]) + fip_pool_obj = FloatingIpPool( + 'user-fip-pool-%s' %(self.id()), parent_obj=vn_obj, + floating_ip_pool_subnets = fip_subnets) + self._vnc_lib.floating_ip_pool_create(fip_pool_obj) + + # Create floating-ip and verify it is from the user-defined subnet. + fip_obj = FloatingIp('user-fip-%s' %(self.id()), fip_pool_obj) + fip_obj.add_project(proj_obj) + self._vnc_lib.floating_ip_create(fip_obj) + fip_obj = self._vnc_lib.floating_ip_read(id=fip_obj.uuid) + fip_addr = fip_obj.get_floating_ip_address() + if ipaddress.ip_address(fip_addr) not in \ + ipaddress.ip_network(u'10.1.1.0/24'): + raise Exception("Floating-ip not allocated from requested subnet") + + # Create a Floating-ip-pool that points to the flat subnet id. + # Verify, floating-ip-pool create succeeds. + fip_subnets = FloatingIpPoolSubnetType(subnet_uuid =[flat_subnet_uuid]) + fip_pool_obj = FloatingIpPool( + 'flat-fip-pool-%s' %(self.id()), parent_obj=vn_obj, + floating_ip_pool_subnets = fip_subnets) + self._vnc_lib.floating_ip_pool_create(fip_pool_obj) + + # Create floating-ip and verify it is from the flat subnet. + fip_obj = FloatingIp('flat-fip-%s' %(self.id()), fip_pool_obj) + fip_obj.add_project(proj_obj) + self._vnc_lib.floating_ip_create(fip_obj) + fip_obj = self._vnc_lib.floating_ip_read(id=fip_obj.uuid) + fip_addr = fip_obj.get_floating_ip_address() + if ipaddress.ip_address(fip_addr) not in \ + ipaddress.ip_network(u'11.1.1.0/24'): + raise Exception("Floating-ip not allocated from requested subnet") + + # end test_floating_ip_alloc + #end class TestIpAlloc if __name__ == '__main__': diff --git a/src/config/api-server/vnc_addr_mgmt.py b/src/config/api-server/vnc_addr_mgmt.py index b3c5f2e31b5..ef10fed8417 100644 --- a/src/config/api-server/vnc_addr_mgmt.py +++ b/src/config/api-server/vnc_addr_mgmt.py @@ -585,6 +585,7 @@ def _get_ipam_subnet_objs_from_ipam_uuid(self, ipam_fq_name, ipam_fq_name_str = ':'.join(ipam_fq_name) subnet_objs = self._subnet_objs.get(ipam_uuid) if subnet_objs is None: + self._subnet_objs[ipam_uuid] = {} #read ipam to get ipam_subnets and generate subnet_objs (ok, ipam_dict) = self._uuid_to_obj_dict('network_ipam', ipam_uuid) diff --git a/src/config/api-server/vnc_cfg_types.py b/src/config/api-server/vnc_cfg_types.py index 1384c4125c9..7efc2622647 100644 --- a/src/config/api-server/vnc_cfg_types.py +++ b/src/config/api-server/vnc_cfg_types.py @@ -211,6 +211,33 @@ def pre_dbe_update(cls, id, fq_name, obj_dict, db_conn, **kwargs): class FloatingIpServer(Resource, FloatingIp): + @classmethod + def _get_fip_pool_subnets(cls, fip_obj_dict, db_conn): + fip_subnets = None + + # Get floating-ip-pool object. + fip_pool_fq_name = fip_obj_dict['fq_name'][:-1] + fip_pool_uuid = db_conn.fq_name_to_uuid('floating_ip_pool', + fip_pool_fq_name) + ok, res = cls.dbe_read(db_conn, 'floating_ip_pool', + fip_pool_uuid) + if ok: + # Successful read returns fip pool. + fip_pool_dict = res + else: + return ok, res + + # Get any subnets configured on the floating-ip-pool. + try: + fip_subnets =\ + fip_pool_dict['floating_ip_pool_subnets'] + except KeyError, TypeError: + # It is acceptable that the prefixes and subnet_list may be absent + # or may be None. + pass + + return True, fip_subnets + @classmethod def pre_dbe_create(cls, tenant_name, obj_dict, db_conn): if obj_dict['parent_type'] == 'instance-ip': @@ -221,9 +248,51 @@ def pre_dbe_create(cls, tenant_name, obj_dict, db_conn): if req_ip and cls.addr_mgmt.is_ip_allocated(req_ip, vn_fq_name): return (False, (400, 'Ip address already in use')) try: - fip_addr = cls.addr_mgmt.ip_alloc_req(vn_fq_name, + # + # Parse through floating-ip-pool config to see if there are any + # guidelines laid for allocation of this floating-ip. + # + fip_subnets = None + ok, ret_val = cls._get_fip_pool_subnets(obj_dict, db_conn) + # On a successful fip-pool subnet get, the subnet list is returned. + # Otherwise, returned value has appropriate reason string. + if ok: + fip_subnets = ret_val + else: + return (ok, (400, "Floating-ip-pool lookup failed with error:"\ + + ret_val)) + + if not fip_subnets: + # Subnet specification was not found on the floating-ip-pool. + # Proceed to allocated floating-ip from any of the subnets + # on the virtual-network. + fip_addr = cls.addr_mgmt.ip_alloc_req(vn_fq_name, asked_ip_addr=req_ip, alloc_id=obj_dict['uuid']) + else: + subnets_tried = [] + # Iterate through configured subnets on floating-ip-pool. + # We will try to allocate floating-ip by iterating through + # the list of configured subnets. + for fip_pool_subnet in fip_subnets['subnet_uuid']: + try: + # Record the subnets that we try to allocate from. + subnets_tried.append(fip_pool_subnet) + + fip_addr = cls.addr_mgmt.ip_alloc_req(vn_fq_name, + sub = fip_pool_subnet, asked_ip_addr=req_ip, + alloc_id=obj_dict['uuid']) + + except cls.addr_mgmt.AddrMgmtSubnetExhausted: + # This subnet is exhausted. Try next subnet. + continue + + if not fip_addr: + # Floating-ip could not be allocated from any of the + # configured subnets. Raise an exception. + raise cls.addr_mgmt.AddrMgmtSubnetExhausted(vn_fq_name, + subnets_tried) + def undo(): db_conn.config_log( 'AddrMgmt: free FIP %s for vn=%s tenant=%s, on undo' @@ -2448,3 +2517,69 @@ def pre_dbe_create(cls, tenant_name, obj_dict, db_conn): def pre_dbe_update(cls, id, fq_name, obj_dict, db_conn, **kwargs): return cls._check_qos_values(obj_dict, db_conn) # end class QosConfigServer + +class FloatingIpPoolServer(Resource, FloatingIpPool): + @classmethod + def pre_dbe_create(cls, tenant_name, obj_dict, db_conn): + # + # Floating-ip-pools corresponding to a virtual-network can be + # 'optionally' configured to be specific to a ipam subnet on a virtual- + # network. + # + # If the subnet is specified, sanity check the config to verify that + # the subnet exists in the virtual-network. + # + + # If subnet info is not specified in the floating-ip-pool object, then + # there is nothing to validate. Just return. + try: + if (obj_dict['parent_type'] != 'virtual-network' or\ + obj_dict['floating_ip_pool_subnets'] is None or\ + obj_dict['floating_ip_pool_subnets']['subnet_uuid'] is None or\ + not obj_dict['floating_ip_pool_subnets']['subnet_uuid']): + return True, "" + except KeyError, TypeError: + return True, "" + + try: + # Get the virtual-network object. + vn_fq_name = obj_dict['fq_name'][:-1] + vn_id = db_conn.fq_name_to_uuid('virtual_network', vn_fq_name) + ok, vn_dict = cls.dbe_read(db_conn, 'virtual_network', vn_id) + if not ok: + return ok, vn_dict + + # Iterate through configured subnets on this FloatingIpPool. + # Validate the requested subnets are found on the virtual-network. + for fip_pool_subnet in \ + obj_dict['floating_ip_pool_subnets']['subnet_uuid']: + vn_ipams = vn_dict['network_ipam_refs'] + subnet_found = False + for ipam in vn_ipams: + if not ipam['attr']: + continue + for ipam_subnet in ipam['attr']['ipam_subnets']: + ipam_subnet_info = None + if ipam_subnet['subnet_uuid']: + if ipam_subnet['subnet_uuid'] != fip_pool_subnet: + # Subnet uuid does not match. This is not the + # requested subnet. Keep looking. + continue + + # Subnet of interest was found. + subnet_found = True + break + + if not subnet_found: + # Specified subnet was not found on the virtual-network. + # Return failure. + msg = "Subnet %s was not found in virtual-network %s" %\ + (fip_pool_subnet, vn_id) + return (False, (400, msg)) + + except KeyError: + return (False, + (400, 'Incomplete info to create a floating-ip-pool')) + + return True, "" +# end class FloatingIpPoolServer diff --git a/src/config/api-server/vnc_db.py b/src/config/api-server/vnc_db.py index a147d90de20..ba846479864 100644 --- a/src/config/api-server/vnc_db.py +++ b/src/config/api-server/vnc_db.py @@ -900,17 +900,23 @@ def iip_update_subnet_uuid(self, iip_dict): return vn_dict = results[0] for ipam in vn_dict.get('network_ipam_refs', []): - subnets = ipam['attr']['ipam_subnets'] - for subnet in subnets: - pfx = subnet['subnet']['ip_prefix'] - pfx_len = subnet['subnet']['ip_prefix_len'] + ipam_subnets = ipam['attr']['ipam_subnets'] + + for ipam_subnet in ipam_subnets: + if 'subnet' not in ipam_subnet or\ + ipam_subnet['subnet'] is None: + # Ipam subnet info need not have ip/prefix info, + # instead they could hold the uuid of subnet info. + continue + pfx = ipam_subnet['subnet']['ip_prefix'] + pfx_len = ipam_subnet['subnet']['ip_prefix_len'] cidr = '%s/%s' % (pfx, pfx_len) if (IPAddress(iip_dict['instance_ip_address']) in IPNetwork(cidr)): - iip_dict['subnet_uuid'] = subnet['subnet_uuid'] + iip_dict['subnet_uuid'] = ipam_subnet['subnet_uuid'] self._object_db.object_update('instance-ip', - iip_dict['uuid'], - iip_dict) + iip_dict['uuid'], + iip_dict) return def _dbe_resync(self, obj_type, obj_uuids): diff --git a/src/container/kube-manager/SConscript b/src/container/kube-manager/SConscript index 7d20a8e28fc..22f3f3af329 100644 --- a/src/container/kube-manager/SConscript +++ b/src/container/kube-manager/SConscript @@ -37,6 +37,7 @@ local_sources = [ 'kube_manager/kube/pod_monitor.py', 'kube_manager/kube/service_monitor.py', 'kube_manager/kube/endpoint_monitor.py', + 'kube_manager/kube/ingress_monitor.py', 'kube_manager/vnc/__init__.py', 'kube_manager/vnc/config_db.py', 'kube_manager/vnc/db.py', @@ -48,6 +49,7 @@ local_sources = [ 'kube_manager/vnc/vnc_pod.py', 'kube_manager/vnc/vnc_service.py', 'kube_manager/vnc/vnc_endpoints.py', + 'kube_manager/vnc/vnc_ingress.py', 'kube_manager/vnc/loadbalancer.py', 'kube_manager/vnc/link_local_manager.py', 'kube_manager/tests/__init__.py', diff --git a/src/container/kube-manager/kube_manager/vnc/config_db.py b/src/container/kube-manager/kube_manager/vnc/config_db.py index b7492f60887..6fe90303458 100644 --- a/src/container/kube-manager/kube_manager/vnc/config_db.py +++ b/src/container/kube-manager/kube_manager/vnc/config_db.py @@ -473,6 +473,8 @@ def update(self, obj=None): self.name = obj['fq_name'][-1] self.fq_name = obj['fq_name'] self.update_single_ref('virtual_network', obj) + if 'floating_ip_pool_subnets' in obj: + self.floating_ip_pool_subnets = obj['floating_ip_pool_subnets'] @classmethod def delete(cls, uuid): diff --git a/src/container/kube-manager/kube_manager/vnc/vnc_kubernetes.py b/src/container/kube-manager/kube_manager/vnc/vnc_kubernetes.py index 24ea5cecd7a..bbda4b0d922 100644 --- a/src/container/kube-manager/kube_manager/vnc/vnc_kubernetes.py +++ b/src/container/kube-manager/kube_manager/vnc/vnc_kubernetes.py @@ -154,12 +154,13 @@ def _get_rule(ingress, sg, prefix, ethertype): def _create_ipam(self, ipam_name, subnets, proj_obj, type='user-defined-subnet'): + ipam_obj = NetworkIpam(name=ipam_name, parent_obj=proj_obj) + ipam_subnets = [] for subnet in subnets: pfx, pfx_len = subnet.split('/') ipam_subnet = IpamSubnetType(subnet=SubnetType(pfx, int(pfx_len))) ipam_subnets.append(ipam_subnet) - ipam_obj = NetworkIpam(name=ipam_name, parent_obj=proj_obj) if type == 'flat-subnet': ipam_obj.set_ipam_subnet_method('flat-subnet') @@ -168,7 +169,7 @@ def _create_ipam(self, ipam_name, subnets, proj_obj, try: self.vnc_lib.network_ipam_create(ipam_obj) except RefsExistError: - vn_obj = self.vnc_lib.network_ipam_read( + ipam_obj = self.vnc_lib.network_ipam_read( fq_name=ipam_obj.get_fq_name()) return ipam_obj, ipam_subnets @@ -176,13 +177,25 @@ def _create_cluster_network(self, vn_name, proj_obj): vn_obj = VirtualNetwork(name=vn_name, parent_obj=proj_obj, address_allocation_mode='user-defined-subnet-only') + # Create Pod IPAM. ipam_obj, ipam_subnets= self._create_ipam('pod-ipam', self.args.pod_subnets, proj_obj) + + # Attach Pod IPAM to virtual-network. vn_obj.add_network_ipam(ipam_obj, VnSubnetsType(ipam_subnets)) - ipam_obj, ipam_subnets = self._create_ipam('service-ipam', + # + # Create Service IPAM. + # + svc_ipam_obj, ipam_subnets = self._create_ipam('service-ipam', self.args.service_subnets, proj_obj, type='flat-subnet') - vn_obj.add_network_ipam(ipam_obj, VnSubnetsType([])) + + # Attach Service IPAM to virtual-network. + # + # For flat-subnets, the subnets are specified on the IPAM and + # not on the virtual-network to IPAM link. So pass an empty + # list of VnSubnetsType. + vn_obj.add_network_ipam(svc_ipam_obj, VnSubnetsType([])) vn_obj.set_virtual_network_properties( VirtualNetworkType(forwarding_mode='l3')) @@ -195,7 +208,7 @@ def _create_cluster_network(self, vn_name, proj_obj): VirtualNetworkKM.locate(vn_obj.uuid) # Create service floating ip pool. - self._create_cluster_service_fip_pool(vn_obj) + self._create_cluster_service_fip_pool(vn_obj, svc_ipam_obj) return vn_obj.uuid @@ -213,27 +226,66 @@ def _get_cluster_service_fip_pool(self): return FloatingIpPoolKM.find_by_name_or_uuid( self._get_cluster_service_fip_pool_name(vn_obj.name)) - def _create_cluster_service_fip_pool(self, vn_obj): - # Create a floating IP pool in cluster service network. + def _create_cluster_service_fip_pool(self, vn_obj, ipam_obj): + # Create a floating-ip-pool in cluster service network. # - # Service IP's in the cluster are allocated from service + # Service IP's in the k8s cluster are allocated from service # IPAM in the cluster network. All pods spawned in isolated - # virtual networks will be allocated an IP from this floating IP - # pool. These pods in those isolated virtual networks will use this - # floating IP for traffic to services in the cluster. + # virtual networks will be allocated an IP from this floating-ip- + # pool. These pods, in those isolated virtual networks, will use this + # floating-ip for outbound traffic to services in the k8s cluster. + + # Get IPAM refs from virtual-network. + ipam_refs = vn_obj.get_network_ipam_refs() + svc_subnet_uuid = None + for ipam_ref in ipam_refs: + if ipam_ref['to'] == ipam_obj.get_fq_name(): + ipam_subnets = ipam_ref['attr'].get_ipam_subnets() + if not ipam_subnets: + import pdb;pdb.set_trace() + continue + # We will use the first subnet in the matching IPAM. + svc_subnet_uuid = ipam_subnets[0].get_subnet_uuid() + break + + fip_subnets = FloatingIpPoolSubnetType(subnet_uuid = [svc_subnet_uuid]) fip_pool_obj = FloatingIpPool( self._get_cluster_service_fip_pool_name(vn_obj.name), + floating_ip_pool_subnets = fip_subnets, parent_obj=vn_obj) try: # Create floating ip pool for cluster service network. - self.vnc_lib.floating_ip_pool_create(fip_pool_obj) - except: - self.logger.error("Service floating-IP-pool create failed for " - "Virtual Network[%s] " % vn_obj.name) - return None + fip_pool_vnc_obj =\ + self.vnc_lib.floating_ip_pool_create(fip_pool_obj) + + except RefsExistError: + # Floating-ip-pool exists. + # + # Validate that existing floating-ip-pool has the service subnet + # uuid as one of its subnets. If not raise an exception, as the + # floating-ip-pool cannot be created, as one with the same name but + # different attributes exists in the system. + fip_pool_vnc_obj = self._get_cluster_service_fip_pool() + svc_subnet_found = False + fip_subnets = None + + if hasattr(fip_pool_vnc_obj, 'floating_ip_pool_subnets'): + fip_subnets = fip_pool_vnc_obj.floating_ip_pool_subnets + + if fip_subnets: + for subnet in fip_subnets['subnet_uuid']: + if subnet == svc_subnet_uuid: + svc_subnet_found = True + break + + if not svc_subnet_found: + self.logger.error("Failed to create floating-ip-pool %s for"\ + "subnet %s. A floating-ip-pool with same name exists." %\ + (":".join(fip_pool_vnc_obj.fq_name), svc_subnet_uuid)) + else: # Update local cache. - FloatingIpPoolKM.locate(fip_pool_obj.uuid) + FloatingIpPoolKM.locate(fip_pool_vnc_obj) return diff --git a/src/container/kube-manager/kube_manager/vnc/vnc_namespace.py b/src/container/kube-manager/kube_manager/vnc/vnc_namespace.py index 417027f3746..0fb64e0fff0 100644 --- a/src/container/kube-manager/kube_manager/vnc/vnc_namespace.py +++ b/src/container/kube-manager/kube_manager/vnc/vnc_namespace.py @@ -74,7 +74,7 @@ def _create_ipam(self, ipam_name, subnets, proj_obj, type): ipam = self._vnc_lib.network_ipam_read(fq_name=ipam_obj.get_fq_name()) # Cache ipam info. - NetworkIpamSM.locate(ipam.uuid) + NetworkIpamKM.locate(ipam.uuid) return ipam_obj, ipam_subnets @@ -98,21 +98,6 @@ def _create_virtual_network(self, ns_name, vn_name, proj_obj, subnets=None): vn_obj = self._vnc_lib.virtual_network_read( fq_name=vn.get_fq_name()) - # Delete existing ipams on this virtual network. - # - # This is so that we can reconcile any possible changes to ipam - # properties/subnet that we may have missed during restart. - # Ipams on the network are self created. So they can be cleaned - # up and recreated, safely. - if vn_obj.get_network_ipam_refs(): - ipam_refs = vn_obj.get_network_ipam_refs() - for ipam in ipam_refs: - ipam_obj = NetworkIpam(name=ipam['to'][-1], - parent_obj=proj_obj) - vn_obj.del_network_ipam(ipam_obj) - self._vnc_lib.virtual_network_update(vn_obj) - self._delete_ipam(ipam) - except NoIdError: # Virtual network does not exist. Create one. self._vnc_lib.virtual_network_create(vn) @@ -182,7 +167,8 @@ def vnc_namespace_add(self, name): # If this namespace is isolated, create it own network. if self._is_namespace_isolated(name) == True: - self._create_virtual_network(ns_name= name, vn_name=name, + vn_name = name + "-vn" + self._create_virtual_network(ns_name= name, vn_name=vn_name, proj_obj = proj_obj) diff --git a/src/container/kube-manager/kube_manager/vnc/vnc_pod.py b/src/container/kube-manager/kube_manager/vnc/vnc_pod.py index 80244d72253..3edcd84d950 100644 --- a/src/container/kube-manager/kube_manager/vnc/vnc_pod.py +++ b/src/container/kube-manager/kube_manager/vnc/vnc_pod.py @@ -71,7 +71,8 @@ def _create_cluster_service_fip(self, pod_name, vmi_obj): # Create Floating-Ip object. fip_obj = FloatingIp(name="cluster-svc-fip-%s"% (pod_name), - parent_obj=fip_pool_obj) + parent_obj=fip_pool_obj, + floating_ip_traffic_direction='egress') fip_obj.set_virtual_machine_interface(vmi_obj) try: